#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :获取懂车帝车型官方图片.py
# @Time      :2024/7/12 
# @Author    :CL
# @email     :1037654919@qq.com
# 实现对懂车帝的爬取
import os
import re
import time
import requests
import urllib3
from DrissionPage import ChromiumPage,ChromiumOptions
from bs4 import BeautifulSoup
from retrying import retry
from tqdm import tqdm
from util import mongo_manager

dongchedi_serires_id = mongo_manager('dongchedi_serires_id', db='car_images')
dongchedi_images_info = mongo_manager('dongchedi_images_info', db='car_images')
urllib3.disable_warnings()  #强制取消警告
co = ChromiumOptions()
co = co.set_argument('--no-sandbox') # 关闭沙箱模式, 解决`$DISPLAY`报错
co = co.set_argument('--headless=new')  # 开启无头模式, 解决`浏览器无法连接`报错
page = ChromiumPage(co)
def get_proxy():
    proxies = {
        'http': 'http://127.0.0.1:15732',
        'https': 'http://127.0.0.1:15732'
    }
    return proxies


headers = {
    'Referer': 'https://www.dongchedi.com/'
}


class Dongchedi:
    def __init__(self):
        self.session = requests.session()
        self.headers  = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Accept-Encoding": "gzip, deflate, br, zstd",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Connection": "keep-alive",
        "Priority": "u=0, i"
    }
        self.cookies = {
        "ttwid": "1%7C0My1NYnVFsoFu9EqFQ5ImURXIWn-55BeWyEXPXH19WI%7C1716537021%7Cd63aa457715bc24a78638a37db8fc62bd21026839c81be634d4ad38af9158d60",
        "tt_webid": "7372470272929367564",
        "Hm_lvt_3e79ab9e4da287b5752d8048743b95e6": "1720747718",
        "_ga": "GA1.2.256978812.1659509211",
        "_ga_YB3EWSDTGF": "GS1.1.1720768914.60.1.1720769079.16.0.0",
        "tt_web_version": "new",
        "s_v_web_id": "verify_lwkdtovt_a91huN3s_mSYH_4Ojg_8QSZ_9uaRPAKcxofN",
        "is_dev": "false",
        "is_boe": "false",
        "city_name": "%E5%8C%97%E4%BA%AC",
        "_gid": "GA1.2.1734826915.1720747718",
        "Hm_lpvt_3e79ab9e4da287b5752d8048743b95e6": "1720769033",
        "HMACCOUNT": "462A68EB325DA0DE"
    }
    #第一步：获取懂车帝车型对应编号
    def get_car_num(self):

        headers = {
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "priority": "u=0, i",
            "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Linux\"",
            "sec-fetch-dest": "document",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "same-origin",
            "sec-fetch-user": "?1",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
        }
        base_url = 'https://www.dongchedi.com/auto/series/{}'
        print('开始获取懂车帝车型对应编号,base_url为：', base_url)
        length = 10200 # 懂车帝汽车数量
        for i in tqdm(range(1, length+1)):
            if dongchedi_serires_id.findOne({'_id': i}):
                continue
            url = base_url.format(i)
            print('开始获取懂车帝车型对应编号：', url)
            try:
                dongchedi_serires_id.insertOne(
                    {'_id': i,  'code': i})
            except Exception as e:
                print(e)
            time.sleep(1)
            try:
                response = requests.get(url, headers=headers, timeout=10)
                html2 = response.text
                soup2 = BeautifulSoup(html2, 'lxml')
                datas = soup2.find('div', class_='new-main tw-overflow-hidden new').find('div',
                                                                                         class_='breadcrumb_wrap__2GLeU').find(
                    'p').find_all('span')
                car_series = datas[2].get_text()
                car_style = datas[3].get_text()
                print('编号：', i, '车系：', car_series, '车型：', car_style)
                try:
                    dongchedi_serires_id.insertOne(
                        {'_id': i, 'car_series': car_series, 'car_style': car_style, 'code': i})
                except Exception as e:
                    dongchedi_serires_id.updateOne({'_id': i},
                        {'_id': i, 'car_series': car_series, 'car_style': car_style, 'code': i})
            except:
                print('没有数据，网址：', url)
                time.sleep(60)

    # 获取图片信息
    @retry(stop_max_attempt_number=5)
    def get_picture_head(self, params=None):
        headers = {
            "accept": "*/*",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "priority": "u=1, i",
            "referer": "https://www.dongchedi.com/auto/series/4363/images-wg",
            "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Linux\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
        }
        cookies = {
            "tt_web_version": "new",
            "ttwid": "1%7CsaLwp8mwFNzMTxnzaeE17hAqmjQpYGtABkXxdOxHYkU%7C1720594973%7C87b4ddc9d166bd502b61baf2fafad5128565bb91da319ec3b6e000df0e47f8f8",
            "tt_webid": "7389899073149978174",
            "is_dev": "false",
            "is_boe": "false",
            "city_name": "%E5%AE%81%E6%B3%A2",
            "Hm_lvt_3e79ab9e4da287b5752d8048743b95e6": "1720594974",
            "HMACCOUNT": "E2603C916CED7269",
            "s_v_web_id": "verify_lyfhtnjn_guogZDed_7AZd_44WE_9SK3_enld2rfBmq6D",
            "_gid": "GA1.2.1893131678.1720594976",
            "Hm_lpvt_3e79ab9e4da287b5752d8048743b95e6": "1720600406",
            "_gat_gtag_UA_138671306_1": "1",
            "_ga_YB3EWSDTGF": "GS1.1.1720598999.7.1.1720600407.60.0.0",
            "_ga": "GA1.1.388278082.1690165116"
        }
        url = "https://www.dongchedi.com/motor/pc/car/series/get_picture_head"
        if params is None:
            params = {
                "aid": "1839",
                "app_name": "auto_web_pc",
                "series_id": "4363"
            }
        response = requests.get(url, headers=headers, cookies=cookies, params=params)

        print(response.url, response)
        return response.json()['data']

    # 获取汽车图片,大类 ，没有详细标签
    @retry(stop_max_attempt_number=3)
    def get_series_picture(self, params=None):
        headers = {
            "accept": "*/*",
            "accept-language": "zh-CN,zh;q=0.9",
            "cache-control": "no-cache",
            "pragma": "no-cache",
            "priority": "u=1, i",
            "referer": "https://www.dongchedi.com/auto/series/4363/images-wg",
            "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Linux\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
        }
        cookies = {
            "tt_web_version": "new",
            "ttwid": "1%7CsaLwp8mwFNzMTxnzaeE17hAqmjQpYGtABkXxdOxHYkU%7C1720594973%7C87b4ddc9d166bd502b61baf2fafad5128565bb91da319ec3b6e000df0e47f8f8",
            "tt_webid": "7389899073149978174",
            "is_dev": "false",
            "is_boe": "false",
            "city_name": "%E5%AE%81%E6%B3%A2",
            "Hm_lvt_3e79ab9e4da287b5752d8048743b95e6": "1720594974",
            "HMACCOUNT": "E2603C916CED7269",
            "s_v_web_id": "verify_lyfhtnjn_guogZDed_7AZd_44WE_9SK3_enld2rfBmq6D",
            "_gid": "GA1.2.1893131678.1720594976",
            "Hm_lpvt_3e79ab9e4da287b5752d8048743b95e6": "1720600406",
            "_gat_gtag_UA_138671306_1": "1",
            "_ga_YB3EWSDTGF": "GS1.1.1720598999.7.1.1720600407.60.0.0",
            "_ga": "GA1.1.388278082.1690165116"
        }
        url = "https://www.dongchedi.com/motor/pc/car/series/get_series_picture"
        if params is None:
            params = {
                "aid": "1839",
                "app_name": "auto_web_pc",
                "series_id": "4363",
                "category": "gft",  #官方
                "offset": "0",
                "count": "5",
                # "city_name": "宁波"
            }
        response = requests.get(url, headers=headers, cookies=cookies, proxies=get_proxy(),params=params)

        print(response.url, response)
        return response.json()['data']

    @retry(stop_max_attempt_number=3)
    def get_color(self,url = "https://www.dongchedi.com/series-96/images/wg-94780-x-1"):
        response = requests.get(url, headers=self.headers, cookies=self.cookies,proxies=get_proxy())
        print('get_color',response.url,response)
        soup =BeautifulSoup(response.text,'lxml')
        data =soup.find('div',class_='filters_colors__2qAUB')
        colors={}
        if data:
            for li in data.find_all('a'):
                name = li.text.strip()
                # 使用正则表达式提取背景色代码
                pattern = r'style="background-color:#([0-9A-Fa-f]{6})"'

                matches = re.findall(pattern, str(li))
                colors[name] ='_'.join(matches)
        print(colors)
        return colors
# 下载图片
@retry(stop_max_attempt_number=3)
def download_picture(url, file):
    headers = {
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "priority": "u=0, i",
        "referer": "https://www.dongchedi.com/",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\"",
        "sec-fetch-dest": "document",
        "sec-fetch-mode": "navigate",
        "sec-fetch-site": "cross-site",
        "sec-fetch-user": "?1",
        "upgrade-insecure-requests": "1",
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    }
    response = requests.get(url,headers=headers)
    print(response.url, response)
    if response.status_code == 200:
        with open(file, 'wb') as f:
            f.write(response.content)
        return 'done'
    return 'error'


# 基于 ChromiumPage 获取webp图片链接（大图）
@retry(stop_max_attempt_number=3)
def get_webp(url=f'https://www.dongchedi.com/series-145/images/gft-77254-x-0'):
    page = ChromiumPage()  #co
    page.refresh()
    page.clear_cache()
    page.get(url)
    # ele_click = page.ele('.head-image_fullscreen__1ePyI').click()
    ele = page.ele('.swiper-slide swiper-slide-active') #swiper-slide gallery-image_slide__VV5AQ
    ele2= page.ele('.gallery-image_root__3nNoR') #swiper-slide gallery-image_slide__VV5AQ
    # print(ele.html)
    # 使用正则表达式匹配 img 标签中的 src 属性
    img_src_pattern = re.compile(r'<img src="(.*?)"')
    matches = img_src_pattern.findall(str(ele.html))
    imgs_src_pattern = re.compile(r'swiper-slide gallery-image_slide__VV5AQ')
    matches2 = imgs_src_pattern.findall(str(ele2.html))
    lens = len(matches2) if matches2 else 3 # 3张图以及以下没有gallery-image
    page.close()
    if matches :
        return f"https:{matches[0]}",lens
    return None, lens


def get_picture_info(seed = None):
    offset = 0
    has_more = True
    while has_more:
        params = {
            "aid": "1839",
            "app_name": "auto_web_pc",
            "series_id": seed['_id'],
            "category": "gft",
            "offset": offset,
            "count": "5",
        }
        pictures = dongchedi.get_series_picture(params)
        if not pictures:
            break
        picture_list = pictures['picture_list']
        has_more = pictures['paging']['has_more']
        if picture_list:
            for picture in picture_list:
                # large_pic_url = picture['large_pic_url']  # 缩略图链接
                print(picture['car_name'],  seed['_id'],picture['car_id'], picture['pic_count'])
                # 信息入库，包括category series_id carid  picture['pic_count'] 用于构造下载webp图的链接
                reslut = {
                    "_id": f"{params['category']}_{params['series_id']}_{picture['car_id']}",
                    "category": params['category'],
                    "series_id": params['series_id'],
                    "car_style": seed['car_style'],
                    "car_id": picture['car_id'],  # 车的id 关键信息
                    "car_name": picture['car_name'],
                    # large_pic_url = picture['large_pic_url'],  # 缩略图链接
                    "year": picture['year'],
                    "base_url": f"https://www.dongchedi.com/series-{params['series_id']}/images/{params['category']}-{picture['car_id']}-x-1",
                    "lens":  picture['pic_count']
                }
                try:
                    dongchedi_images_info.insertOne(reslut)
                except Exception as e:
                    dongchedi_images_info.updateOne({"_id": reslut["_id"]},reslut)
        offset += 5


def main():
    while True:
        seed = dongchedi_serires_id.find_one_and_update({"status": None}, {"status": 'running'})
        if seed:
            get_picture_info(seed)
            seed['status'] = 'done'
            dongchedi_serires_id.updateOne({"_id": seed['_id']}, seed)
        else:
            break

def main2():
    # dongchedi_images_info.updateMany({"status": "running"}, {"status": None})
    # todo 完善jpg的下载
    while True:
        seed = dongchedi_images_info.find_one_and_update({"status": None}, {"status": 'running'})
        # step3 下载图片
        if seed:
            url = seed['base_url']
            colors = dongchedi.get_color(url)
            seed['colors'] = colors
            seed['webp_urls']={}
            if colors:
                for color,code in colors.items():

                    code = code.replace('#','')
                    if "_" not in code:
                        code = f"{code}_"
                    url = f"https://www.dongchedi.com/series-{seed['series_id']}/images/{seed['category']}-{seed['car_id']}-{code}-0"
                    webp_url,lens = get_webp(url)
                    if webp_url:
                        print(webp_url)
                        seed['webp_urls'][url] = webp_url
                        seed['colors'][color] = f"{code}_{lens}"
                        # download_picture(webp_url,file=f"{path}{webp_url.split('/')[-1].split('?')[0]}")
                    for num in range(1, lens):
                        url = f"https://www.dongchedi.com/series-{seed['series_id']}/images/{seed['category']}-{seed['car_id']}-{code}-{num}"
                        webp_url,lens = get_webp(url)
                        if webp_url:
                            print(webp_url)
                            seed['webp_urls'][url] = webp_url
                            # download_picture(webp_url, file=f"{path}{webp_url.split('/')[-1].split('?')[0]}")
            else:

                url = f"https://www.dongchedi.com/series-{seed['series_id']}/images/{seed['category']}-{seed['car_id']}-x-0"
                webp_url, lens = get_webp(url)
                if webp_url:
                    seed['webp_urls'][url] = webp_url
                    print(webp_url)
                    # download_picture(webp_url, file=f"{path}{webp_url.split('/')[-1].split('?')[0]}")
                for num in range(1, lens):
                    url = f"https://www.dongchedi.com/series-{seed['series_id']}/images/{seed['category']}-{seed['car_id']}-x-{num}"
                    webp_url, lens = get_webp(url)
                    if webp_url:
                        print(webp_url)

                        seed['webp_urls'][url] = webp_url
                        # download_picture(webp_url, file=f"{path}{webp_url.split('/')[-1].split('?')[0]}")

            seed['status'] = 'nodown'

            dongchedi_images_info.updateOne({"_id": seed['_id']}, seed)
        else:
            break

if __name__ == '__main__':
    print()
    dongchedi = Dongchedi()
    # # step1 获取 车型id
    # datas = dongchedi.get_car_num()
    # # step2 获取车型图片数量 信息
    # dongchedi_serires_id.updateMany({}, {"status": None})
    # main()
    # # step3 下载图片链接
    page = ChromiumPage()
    main2()
    page.close()
